# -*- coding: utf-8 -*-
'''
Created on 07.11.2013

@author: Liesa
'''

import re
import glob
import os
import trunk.data
from collections import defaultdict
from io import open
import sys

class Corpus_Reader:
    """class for reading in data"""
    def __init__(self):
        self.count = 0
        
        self.transcribed = [] #\t
        self.morphemes = [] #\m
        self.pos_tags = [] #\c
        self.gloss = [] #\g
        
        self.prefixes = dict()
        self.suffixes = dict()
        self.stems = set()
        self.stemstrans = defaultdict(set)
        self.transstems = defaultdict(set)
        
        self.verbs = dict()
        self.spanish = set()
    
    def read_in(self):
        "function for reading in the files"
        mypath = os.path.join(trunk.data.__path__[0], "usp", "tb")
        os.chdir(mypath)
        for files in glob.glob("*.final"):
            #print(files)
            with open(files, encoding="utf8") as f:
                for line in f:
                    #transcribed line
                    if line.startswith("\\t "):
                        self.transcribed = re.sub(ur'[\d:/,.?!+\xbf\xa1\(\)]+', "", line.strip()[3:len(line)-2]).lower().split()
      
                    #line with morphemes                
                    elif line.startswith("\\m "):
                        self.morphemes = line[3:].lower().split()                        
                    
                    #line with pos-tags
                    elif line.startswith("\\c "):
                        self.pos_tags = line[3:].split()

                    #line with glosses
                    elif line.startswith("\\g "):
                        self.gloss = line[3:].split()
                    
                    #after block of lines
                    elif line == '\n':
                        #print("\nTranscribed:", self.transcribed)
                        #print("\nMorphemes:", self.morphemes)
                        #print(("Pos-Tags:"), self.pos_tags)
                        self.count_words()
                        self.get_morphemes()
                        self.get_verbs()
                        self.get_spanish()
                    
                    
       
    def count_words(self):
        "function for counting words"
        self.count += len(self.transcribed)
    
    def get_morphemes(self):
        "get the pos tag of the stem of the affix"
        tagpattern = re.compile(r"[A-Z]+[1-9/]*[A-Z]+")

        
        for i in range(0, len(self.morphemes)):
            morph = self.morphemes[i]
            
            
            #prefixe BLABLA-
            if morph[-1] == '-':
                #print(morph)
                for j in range(i, len(self.morphemes)):
                    if "-" not in self.morphemes[j]:
                        #print("--->", self.morphemes[j], self.pos_tags[j])
                        pos = self.pos_tags[j]
                        try:
                            self.prefixes[morph][pos] = self.prefixes[morph].get(pos, 0) + 1
                        except:
                            self.prefixes[morph] = dict()
                            self.prefixes[morph][pos] = 1
                        break
            
            #suffixe  -BLABLA
            elif morph[0] == '-':
                #print(morph)
                for j in range(i, 0, -1):
                    if "-" not in self.morphemes[j]:
                        #print("--->", self.morphemes[j], self.pos_tags[j])
                        pos = self.pos_tags[j]
                        try:
                            self.suffixes[morph][pos] = self.suffixes[morph].get(pos, 0) + 1
                        except:
                            self.suffixes[morph] = dict()
                            self.suffixes[morph][pos] = 1
                        break
            #stems
            else:
                pos = self.pos_tags[i]
                self.stems.add((morph,pos))
                if re.search(tagpattern, self.gloss[i]) or self.gloss[i] == 'V' or self.gloss[i] == 'S' or self.gloss[i] == '???':
                    continue
                else:
                    self.stemstrans[morph].add(self.gloss[i])
                    self.transstems[self.gloss[i]].add(morph)
                
    def get_morphemes1(self):
        for i in range(0, len(self.morphemes)-1):
            morph = self.morphemes[i]
            pos = self.pos_tags[i]
            
            if pos == "???" or morph == "???":
                continue
            if morph[-1] == '-':
                self.prefixes.add((morph,pos))
            elif morph[0] == '-':
                self.suffixes.add((morph,pos))
            else:
                self.stems.add((morph,pos))

    def get_verbs(self):
        """Method for finding and mapping the spanish translation to all the 
        occuring forms for that verb"""
        
        i = 0
        while i < len(self.pos_tags):
            tag = self.pos_tags[i]
            if tag.startswith("V") and tag != "VOC":
                translation = self.gloss[i]
                
                real = 0
                j=0
                while j <= i:
                    word = self.morphemes[j]
                    if word[0] != '-' and word[-1] != "-":
                        real += 1
                    j+=1
                
                try:
                    verb = self.transcribed[real-1]
                    try:
                        self.verbs[translation].add(verb)
                    except:
                        self.verbs[translation] = {verb}
                except:
                    pass
            i+=1
    
    def read_translations(self):
        translations = dict()
        mypath = trunk.data.__path__[0]
        with open(os.path.join(mypath, 'translations.txt'), encoding='utf8') as f:
            for line in f:
                line = line.split("||")
                if len(line) >= 3:
                    spanish = line[0].strip().lower()
                    uspanteko = line[1].strip().lower()
                    translations[spanish] = uspanteko
        return translations
    
    def save_verbs(self):
        """Method for saving the verb table in a file
        Saves it like this:
        """
        trans_dict = self.read_translations()
        there = 0
        not_there = 0
        
        mypath = trunk.data.__path__[0]
        with open(os.path.join(mypath, 'verb_table.txt'), 'w', encoding='utf8') as f:
            for translation in self.verbs:
                try:
                    verb = trans_dict[translation.lower()]
                    there += 1
                except:
                    not_there += 1
                    continue
            
                f.write("%s\t" % verb)
                for stuff in self.verbs[translation]:
                    f.write("%s " % stuff)
                f.write(u"\n")
        
        print "\nMaking Verb Tables:"       
        print "Not there:", not_there
        print "There:", there
    
    def save(self):
        """Saves affixes to files"""
        mypath = trunk.data.__path__[0]
        with open(os.path.join(mypath, 'prefixes.txt'), 'w', encoding='utf8') as f:
            for pref in self.prefixes:
                #for pos in self.prefixes[pref]:
                f.write("%s\t%s\n" % (pref, self.prefixes[pref]))
        with open(os.path.join(mypath, 'suffixes.txt'), 'w', encoding='utf8') as f:
            for suf in self.suffixes:
                #for pos in self.suffixes[suf]:
                f.write("%s\t%s\n" % (suf,self.suffixes[suf]))
        with open(os.path.join(mypath, 'stems.txt'), 'w', encoding='utf8') as f:
            for (stem, pos) in self.stems:
                f.write("%s/%s\n" % (stem,pos))
        print "Writing morphemes to files done."
    
    def get_spanish(self):
        """Finds and collects the Spanish loanwords in Uspanteko:
        Compared the words from \m and \g line with each other, saves them
        if they are exactly the same"""
        for i in range(0, len(self.morphemes)-1):
            morph = self.morphemes[i]
            glosses = self.gloss[i].lower()
            
            if morph == glosses:
                self.spanish.add(morph)
    
    def save_spanish(self):
        """Saves spanish words to file"""
        mypath = trunk.data.__path__[0]
        with open(os.path.join(mypath, 'spanish.txt'), 'w', encoding='utf8') as f:
            for x in self.spanish:
                if x != "???":
                    f.write("%s\n" % x)
            
                
def main():
    l = Corpus_Reader()
    l.read_in() 
    l.save()   
    l.save_verbs()
    l.save_spanish()

if __name__ == '__main__':
    main()
